package com.chance.cc.crawler.development.bootstrap.autohome.forum;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import com.chance.cc.crawler.development.utils.RedisReader;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;

import java.io.FileInputStream;
import java.io.IOException;
import java.util.List;

import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.Tag_Site_Info;

/**
 * @author lt
 * @version 1.0
 * @date 2020-11-18 10:07:09
 * @email okprog@sina.com
 */
public class ForumReplyStart {

    private static final String domain = "autohome";

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }
    HttpConfig httpConfig = HttpConfig.me(domain);

    public static void main(String[] args) {
        pushRecord2Redis();

    }


    public static void pushRecord2Redis(){
        String site = "forum_reply";
        String filePath = "C:\\Users\\Administrator\\Documents\\chance\\chance-crawler-development\\crawler-dev-bootstrap\\src\\main\\java\\com\\chance\\cc\\crawler\\development\\bootstrap\\autohome\\forum\\forum_url.json";
        RedisReader redisReader = new RedisReader("192.168.1.215",6379,4);
        try {
            List<String> allUrls = IOUtils.readLines(new FileInputStream(filePath),"utf-8");
            for (String allUrl : allUrls) {
                JSONObject urlJSON = JSONObject.parseObject(allUrl);
                String itemUrl = urlJSON.getString("itemUrl");
                String title = urlJSON.getString("title");
                String author = urlJSON.getString("author");
                String authorId = urlJSON.getString("authorId");
                String isVideo = urlJSON.getString("isVideo");
                String topicId = itemUrl.substring(itemUrl.lastIndexOf("/") + 1).split("-")[0];

                CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                        .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPageItem)
                        .httpUrl(itemUrl)
                        .releaseTime(System.currentTimeMillis())
                        .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                        .addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 32,null))
                        .needParsed(true)
                        .needWashed(false)
                        .proxy(proxy)
                        .build();

                requestRecord.tagsCreator().bizTags().addDomain(domain);
                requestRecord.tagsCreator().bizTags().addSite(site);
                requestRecord.tagsCreator().bizTags().addCustomKV(Tag_Site_Info,"forum-adhoc-0608");

                HttpRequest httpRequest = requestRecord.getHttpRequest();
                httpRequest.addExtra("topicId",topicId);
                httpRequest.addExtra("title",title);
                httpRequest.addExtra("author",author);
                httpRequest.addExtra("authorId",authorId);
                httpRequest.addExtra("isVideo",isVideo);
                httpRequest.addExtra("itemUrl",itemUrl);

                httpRequest.addHeader("Host","club.autohome.com.cn");
                httpRequest.addHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");

                String requestKey = requestRecord.getRecordKey();
                //确保key值唯一
                String shaKey = DigestUtils.sha1Hex(System.currentTimeMillis()+requestKey);

                redisReader.mapPush("crawler-autohome-forum_reply-simple-queue_map",shaKey, JSON.toJSONString(requestRecord));
                redisReader.listPush("crawler-autohome-forum_reply-simple-queue_list",shaKey);

            }

        } catch (IOException e) {
            e.printStackTrace();
        }


    }
}
